# plotly standard imports
import plotly.graph_objs as go
import chart_studio.plotly as py

# Cufflinks wrapper on plotly
import cufflinks

# Data science imports
import pandas as pd
import numpy as np

# Options for pandas
pd.options.display.max_columns = 30

# Display all cell outputs
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = 'all'

from plotly.offline import iplot, init_notebook_mode
cufflinks.go_offline(connected=True)
init_notebook_mode(connected=True)

# Set global theme
cufflinks.set_config_file(world_readable=True, theme='pearl')

Let's explore datasets

Explore input dataset

Will use target dataset Bitcoin Historical Data

Bitcoin data at 1-min intervals from select exchanges, Jan 2012 to Dec 2020

from src.load_datasets import load_input_dataset

input_dataset = load_input_dataset()

input_dataset.head()

	Timestamp	Open	High	Low	Close	Volume_(BTC)	Volume_(Currency)	Weighted_Price
0	1325317920	4.39	4.39	4.39	4.39	0.455581	2.0	4.39
1	1325317980	NaN	NaN	NaN	NaN	NaN	NaN	NaN
2	1325318040	NaN	NaN	NaN	NaN	NaN	NaN	NaN
3	1325318100	NaN	NaN	NaN	NaN	NaN	NaN	NaN
4	1325318160	NaN	NaN	NaN	NaN	NaN	NaN	NaN

Will explore full input dataset, some values contain NaN, which not ineraptebale by sweetviz, so will use timestamp as target feature for now

import sweetviz as sv

analyse_report = sv.analyze([input_dataset, 'Input'], target_feat="Timestamp")
analyse_report.show_notebook()

Will take one timestamp per hour for faster interpretation

input_dataset = input_dataset[59::60]
raw_timestamps = input_dataset.pop('Timestamp')

timestamp need interprate as date for charts processing

input_datetime = pd.to_datetime(raw_timestamps, unit='s')

input_dataset.head()

	Open	High	Low	Close	Volume_(BTC)	Volume_(Currency)	Weighted_Price
59	NaN	NaN	NaN	NaN	NaN	NaN	NaN
119	NaN	NaN	NaN	NaN	NaN	NaN	NaN
179	NaN	NaN	NaN	NaN	NaN	NaN	NaN
239	NaN	NaN	NaN	NaN	NaN	NaN	NaN
299	NaN	NaN	NaN	NaN	NaN	NaN	NaN

Feature evalution over time

input_features = input_dataset[['Open', 'Close', 'Weighted_Price']]
input_features.index = input_datetime

input_features.iplot(
    subplots=True,
)

input_dataset.describe().transpose()

	count	mean	std	min	25%	50%	75%	max
Open	57539.0	4597.869997	4781.860481	4.230000	436.000000	3308.83000	8178.790000	2.892734e+04
High	57539.0	4600.855412	4785.273937	4.230000	436.335000	3312.82000	8183.195000	2.897090e+04
Low	57539.0	4594.739766	4778.247471	4.230000	435.900000	3304.00000	8173.460000	2.892734e+04
Close	57539.0	4597.868472	4781.911473	4.230000	436.025000	3305.00000	8177.300000	2.897090e+04
Volume_(BTC)	57539.0	9.003106	29.133884	0.000001	0.370362	1.85132	6.922594	1.540919e+03
Volume_(Currency)	57539.0	31184.490106	105810.113064	0.000178	411.516416	3109.00990	20379.086506	4.865723e+06
Weighted_Price	57539.0	4597.865954	4781.891626	4.230000	436.030107	3305.86258	8177.593176	2.895104e+04

Will take only last three yers, because they have data without missing values

day = 24
year = (365)*day

input_dataset = input_dataset.tail(3 * year)
input_datetime = input_datetime.tail(3 * year)

input_dataset.head()
len(input_datetime)

	Open	High	Low	Close	Volume_(BTC)	Volume_(Currency)	Weighted_Price
3151019	13633.25	13670.99	13631.95	13631.95	0.871489	11882.568302	13634.789405
3151079	13460.87	13489.55	13460.60	13489.55	8.170318	110043.490480	13468.690911
3151139	13320.00	13350.00	13303.09	13350.00	10.190223	135747.508220	13321.347592
3151199	13540.81	13611.64	13527.87	13527.87	11.510917	156584.137430	13603.098257
3151259	13616.00	13640.59	13606.68	13640.59	3.617158	49269.986162	13621.187176

0

input_features = input_dataset[['Open', 'Close', 'Weighted_Price']]
input_features.index = input_datetime

input_features.iplot(
    subplots=True,
)

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-19-7bbc70516061> in <module>
      1 input_features = input_dataset[['Open', 'Close', 'Weighted_Price']]
----> 2 input_features.index = input_datetime
      3 
      4 input_features.iplot(
      5     subplots=True,

/usr/local/lib/python3.6/dist-packages/pandas/core/generic.py in __setattr__(self, name, value)
   5152         try:
   5153             object.__getattribute__(self, name)
-> 5154             return object.__setattr__(self, name, value)
   5155         except AttributeError:
   5156             pass

pandas/_libs/properties.pyx in pandas._libs.properties.AxisProperty.__set__()

/usr/local/lib/python3.6/dist-packages/pandas/core/generic.py in _set_axis(self, axis, labels)
    562     def _set_axis(self, axis: int, labels: Index) -> None:
    563         labels = ensure_index(labels)
--> 564         self._mgr.set_axis(axis, labels)
    565         self._clear_item_cache()
    566 

/usr/local/lib/python3.6/dist-packages/pandas/core/internals/managers.py in set_axis(self, axis, new_labels)
    225         if new_len != old_len:
    226             raise ValueError(
--> 227                 f"Length mismatch: Expected axis has {old_len} elements, new "
    228                 f"values have {new_len} elements"
    229             )

ValueError: Length mismatch: Expected axis has 26280 elements, new values have 0 elements

Training data distribution

train_df = pd.DataFrame(tfds.as_numpy(train_data), columns=['text', 'type'])

train_df['type'] = train_df['type'].apply(humanize_label)

train_df.head()

N/A% (0 of 1600) |                       | Elapsed Time: 0:00:00 ETA:  --:--:--

Start reading dataset from ./data/training.1600000.processed.noemoticon.csv

100% (1600 of 1600) |####################| Elapsed Time: 0:07:46 Time:  0:07:46

	text	type
0	b"@switchfoot http://twitpic.com/2y1zl - Awww,...	bad
1	b"is upset that he can't update his Facebook b...	bad
2	b'@Kenichan I dived many times for the ball. M...	bad
3	b'my whole body feels itchy and like its on fi...	bad
4	b"@nationwideclass no, it's not behaving at al...	bad

print('Training dataset records', len(train_df.index))

train_df['type'].iplot(
    kind='hist',
    yTitle='count',
    xTitle='Type',
    title='Training data distribution'
)

Training dataset records 1600000

Testing data distribution

test_df = pd.DataFrame(tfds.as_numpy(test_data), columns=['text', 'type'])

test_df['type'] = test_df['type'].apply(humanize_label)

test_df[30:40]

Start reading dataset from ./data/testdata.manual.2009.06.14.csv

	text	type
30	b'need suggestions for a good IR filter for my...	neutral
31	b'@surfit: I just checked my google for my bus...	neutral
32	b"@phyreman9 Google is always a good place to ...	good
33	b'Played with an android google phone. The sli...	bad
34	b'US planning to resume the military tribunals...	bad
35	b'omg so bored & my tattoooos are so itchy...	bad
36	b"I'm itchy and miserable!"	bad
37	b"@sekseemess no. I'm not itchy for now. Maybe...	bad
38	b'RT @jessverr I love the nerdy Stanford human...	good
39	b'@spinuzzi: Has been a bit crazy, with steep ...	good

print('Testing dataset records', len(test_df.index))

neutralSeries = test_df.apply(lambda x: True if x['type'] == 'neutral' else False, axis=1)
print('Count of neutral rows', len(neutralSeries[neutralSeries == True].index))

test_df['type'].iplot(
    kind='hist',
    yTitle='count',
    xTitle='Type',
    title='Testing data distribution'
)

Testing dataset records 498
Count of neutral rows 139

Check preprocessed training datasets distribution

train_prep_df = pd.DataFrame(tfds.as_numpy(train_prep_dataset), columns=['text', 'type'])

train_prep_df['type'] = train_prep_df['type'].apply(humanize_label)

train_prep_df.head()

N/A% (0 of 1600) |                       | Elapsed Time: 0:00:00 ETA:  --:--:--

Start reading dataset from ./preprocessed/training.1600000.processed.noemoticon.csv

100% (1600 of 1600) |####################| Elapsed Time: 0:06:58 Time:  0:06:58

	text	type
0	b'username link - www bummer . you should get ...	bad
1	b'upset he can not update his Facebook texting...	bad
2	b'username i dive time ball . manage save ## r...	bad
3	b'my body feel itchy and like fire'	bad
4	b'username no it not behave at all . i mad . w...	bad

print('Training dataset records', len(train_prep_df.index))

train_prep_df['type'].iplot(
    kind='hist',
    yTitle='count',
    xTitle='Type',
    title='Preprocessed training data distribution'
)

Training dataset records 1599976

Check testing dataset

test_prep_df = pd.DataFrame(tfds.as_numpy(test_prep_dataset), columns=['text', 'type'])

test_prep_df['type'] = test_prep_df['type'].apply(humanize_label)

test_prep_df.head()

Start reading dataset from ./preprocessed/testdata.manual.2009.06.14.csv

	text	type
0	b'username i loooooooovvvvvveee my Kindle # . ...	good
1	b'read my kindle # ... love it ... Lee childs ...	good
2	b'okay \xc3\xaf\xc2\xbb\xc2\xbf # assessment k...	good
3	b'username you will love Kindle # . i month an...	good
4	b'username Fair enough . but i Kindle # and i ...	good

print('Training dataset records', len(test_prep_df.index))

test_prep_df['type'].iplot(
    kind='hist',
    yTitle='count',
    xTitle='Type',
    title='Preprocessed testing data distribution'
)

Training dataset records 498

Explore training metrics

df = pd.read_csv('./metrics/training.csv')
df.head()

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-5-30480f2a5022> in <module>
----> 1 df = pd.read_csv('./metrics/training.csv')
      2 df.head()

/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in read_csv(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)
    684     )
    685 
--> 686     return _read(filepath_or_buffer, kwds)
    687 
    688 

/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in _read(filepath_or_buffer, kwds)
    450 
    451     # Create the parser.
--> 452     parser = TextFileReader(fp_or_buf, **kwds)
    453 
    454     if chunksize or iterator:

/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in __init__(self, f, engine, **kwds)
    934             self.options["has_index_names"] = kwds["has_index_names"]
    935 
--> 936         self._make_engine(self.engine)
    937 
    938     def close(self):

/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in _make_engine(self, engine)
   1166     def _make_engine(self, engine="c"):
   1167         if engine == "c":
-> 1168             self._engine = CParserWrapper(self.f, **self.options)
   1169         else:
   1170             if engine == "python":

/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in __init__(self, src, **kwds)
   1996         kwds["usecols"] = self.usecols
   1997 
-> 1998         self._reader = parsers.TextReader(src, **kwds)
   1999         self.unnamed_cols = self._reader.unnamed_cols
   2000 

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.__cinit__()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._setup_parser_source()

FileNotFoundError: [Errno 2] No such file or directory: './metrics/training.csv'

df[['epoch', 'accuracy', 'val_accuracy']].iplot(
    x='epoch',
    mode='lines+markers',
    xTitle='epoch',
    yTitle='accuracy', 
    title='Training accuracy',
    linecolor='black',
)

df[['epoch', 'loss', 'val_loss']].iplot(
    x='epoch',
    mode='lines+markers',
    xTitle='epoch',
    yTitle='accuracy', 
    title='Losses'
)

Predictions

Load probability model

which can give predictions on model classes

0 - bad review, 1 - good revie

from src.predict import get_probability_model

model = get_probability_model()

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
bidirectional (Bidirectional (None, None, 128)         186880    
_________________________________________________________________
bidirectional_1 (Bidirection (None, 64)                41216     
_________________________________________________________________
dense (Dense)                (None, 64)                4160      
_________________________________________________________________
dropout (Dropout)            (None, 64)                0         
_________________________________________________________________
dense_1 (Dense)              (None, 2)                 130       
=================================================================
Total params: 232,386
Trainable params: 232,386
Non-trainable params: 0
_________________________________________________________________

Firstly will try predict on some data from training dataset

from src.predict import get_text_and_label_from_dataset, predict
REVIEW_INDEX = 110

text, real_label = get_text_and_label_from_dataset(REVIEW_INDEX)

print('text for prediction\n\n', text, '\n')

predicted_label, predictions = predict(text, model)

print(label_categories[predicted_label], 'review')

print('\n\nPredicted label:', predicted_label, 'real label: ', real_label, 'predictions:', predictions)
if (predicted_label == real_label):
    print('Successfully predicted')
else:
    print('Failed to predict')

text for prediction

 Dakota (1988) was another early Lou Diamond Phillips starring vehicle. This film is similar to the later released film Harley. There are a few differences but they're both the same. I don't know which one came first. I guess it'll remain one of the mysteries of life. But they both are troubled "kids" who are trying to turn there lives around. Instead of bikes this one involves horses. They're basically the same movie and they're both cheesy as hell. If you're a serious L.D.P. fan then I recommend that you watch them both. You get some extreme mugging and posturing from L.D.P. if you're game then go for it.<br /><br />Not recommended, except for L.D.P. fans!!! 

bad review


Predicted label: 0 real label:  0 predictions: [0.70361704 0.29638302]
Successfully predicted

Then will try predict hadnwritten text

# Can change text and check model
hadwriten = 'This is good film'

print('Hendwriten text:\n', hadwriten, '\n')

handwriten_label, predictions = predict(hadwriten, model)

print(label_categories[predicted_label], 'review')

print('Probabilities', predictions)

Hendwriten text:
 This is good film 

bad review
Probabilities [0.3463477 0.6536523]